package edu.kit.ksri.db.mturk.plugins.correctness;

import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.log4j.Logger;

import com.amazonaws.mturk.requester.Assignment;

import edu.kit.ksri.db.mturk.connectionhandlers.HITManager;
import edu.kit.ksri.db.mturk.plugins.correctness.answers.AssignmentAnswerFactory;
import edu.kit.ksri.db.mturk.plugins.correctness.answers.ICDAtomicAnswer;
import edu.kit.ksri.db.mturk.plugins.correctness.answers.ICDCompositeAssignmentAnswer;
import edu.kit.ksri.db.mturk.utils.messaging.Result;
import edu.kit.ksri.db.mturk.utils.messaging.ResultOutputCollector;
import edu.kit.ksri.db.mturk.var.Icd;
import edu.kit.ksri.db.mturk.var.StaticInformation;
import edu.kit.ksri.db.mturk.workerpools.ICDWorker;
import edu.kit.ksri.db.mturk.workerpools.PoolManager;

/**
 * Copyright (c) 2010-2011 Karlsruhe Institute of Technology (KIT), Germany
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 * 
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 * 
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 * 
 * --------------------------------------------------------------------------
 * 
 * Software: CSP/WMV tool for dynamic quality management of microtasks
 * http://www.ksri.kit.edu/Upload/Users/PeopleClouds/HCOMP2011_An_extendable_toolkit_for_managing_quality_of_human-based_eServices.pdf
 * 
 * Organization: Karlsruhe Institute of Technology (KIT), Germany
 * http://www.kit.edu
 * 
 * Project: PeopleClouds project of the Karlsruhe Service Research Institute(KSRI)
 * http://www.ksri.kit.edu/Default.aspx?PageId=818
 * 
 * Contributing authors: David Bermbach, Sandra Rath, Pascal Wichmann, Robert Kern
 * 
 * ---------------------------------------------------------------------------
 * 
 * Purpose: This class implements a comparison of label subsets and can also be
 *         used in combination with the CSP-1
 */
public class ICDWMV extends AbstractCSP1AndWMVImplementation {

	private static Logger basiclog = Logger.getLogger("basic." + ICDWMV.class);

	@Override
	protected void initWMV(final String hitID) {

		Result r = ResultOutputCollector.getInstance().get(hitID);
		basiclog.info("Initialising multi-labelling CSP1/WMV for HIT " + hitID
				+ " with " + r.getAssignments().size() + " assignments.");

		double phiMin = StaticInformation.scenario
				.getMinimumInspectionQualityLevel();
		double epsilonMax = StaticInformation.scenario.getEscalationLimit();

		Result newResult = ResultOutputCollector.getInstance().get(hitID);
		basiclog.info("We now have " + newResult.getAssignments().size()
				+ " assignment(s) for HIT " + hitID + ".");

		// put references to all participating workers in an array

		ICDWorker[] workers = null;

		if (StaticInformation.scenario instanceof Icd) {
			workers = new ICDWorker[newResult.getAssignments().size()];
		} else {
			basiclog.error("No appropriate worker class found! ICD worker are required.");
		}

		int counter = 0;
		for (Assignment a : newResult.getAssignments()) {
			workers[counter] = (ICDWorker) PoolManager.getInstance()
					.getPoolForName("ACTIVE").getWorkerForID(a.getWorkerId());
			if (workers[counter] == null) {
				// must be in pending or blocked pool now
				workers[counter] = (ICDWorker) PoolManager.getInstance()
						.getPoolForName("PENDING")
						.getWorkerForID(a.getWorkerId());
				if (workers[counter] == null) {
					// must be in blocked pool now
					workers[counter] = (ICDWorker) PoolManager.getInstance()
							.getPoolForName("BLOCKED")
							.getWorkerForID(a.getWorkerId());
					if (workers[counter] == null) {
						// what shall we do if we cannot find our work?
						// disregard the result?
						// FIXME
					}
				}
			}
			counter++;
		}

		/*
		 * arrays for labelling performance are created
		 */
		double[] specificity = new double[workers.length];
		double[] sensitivity = new double[workers.length];

		counter = 0;
		for (ICDWorker i : workers) {
			specificity[counter] = i.getAverageSpecificity();
			sensitivity[counter] = i.getAverageSensitivity();
			basiclog.info("After " + i.getTotalAssignments()
					+ " assignment(s) Worker " + (counter + 1)
					+ " has a historical average sensitivity of "
					+ sensitivity[counter]
					+ " and a historical average specificity of "
					+ specificity[counter] + ".");
			counter++;
		}

		/*
		 * Each worker has submitted an assignment. These assignments are
		 * transformed into ICDCompositeAssignmentAnswers and stored in an array
		 * of such objects. To keep the link between
		 * ICDCompositeAssignmentAnswer and the corresponding worker who
		 * submitted it the position of the object is EXACTLY the position of
		 * the worker in the worker array: answers[0] was submitted by
		 * workers[0] etc.
		 */
		List<Assignment> assignments = r.getAssignments();
		ICDCompositeAssignmentAnswer[] answers = new ICDCompositeAssignmentAnswer[workers.length];
		for (int i = 0; i < assignments.size(); i++) {
			for (int j = 0; j < workers.length; j++) {
				if (workers[j].getWorkerId().equals(
						assignments.get(i).getWorkerId())) {
					answers[j] = AssignmentAnswerFactory
							.createICDCompositeAssignmentAnswerInstance(assignments
									.get(i).getAnswer());
				}
			}
		}

		/*
		 * At this point we deal with ICDCompositeAssignmentAnswers (i.e.
		 * multiple labels). To compare on a label-basis, we need the atomic
		 * answers contained in each CompositeAnswer.
		 * 
		 * A new HashSet is created to contain the AtomicAnswers of all workers.
		 * The idea is just to let the HashSet automatically remove duplicates
		 * in the set of returned AtomicAnswers (therefore "distinct").
		 */
		Set<ICDAtomicAnswer> allDistinctAtomicAnswers = new HashSet<ICDAtomicAnswer>();

		/*
		 * AtomicAnswers are already stored as HashSets in the CompositeAnswer.
		 * All that needs to be done now is to get the HashSet for each
		 * CompositeAnswer and add the elements to the allDistinctAtomicAnswers
		 * HashSet
		 */
		for (ICDCompositeAssignmentAnswer caa : answers) {
			allDistinctAtomicAnswers.addAll(caa.getSetOfAtomicAnswers());
			for (ICDAtomicAnswer icdaa : caa.getSetOfAtomicAnswers()) {
				icdaa.printMe();
			}

		}

		basiclog.info("The CSP-1/WMV considers "
				+ allDistinctAtomicAnswers.size()
				+ " distinct label(s) assigned by any worker.");

		/*
		 * Because a fixed order of AtomicAnswers is required later the HashSet
		 * is converted into an array of AtomicAnswers.
		 */
		ICDAtomicAnswer[] allDistinctAtomicAnswersArray = allDistinctAtomicAnswers
				.toArray(new ICDAtomicAnswer[allDistinctAtomicAnswers.size()]);

		for (int i = 0; i < allDistinctAtomicAnswersArray.length; i++) {
			allDistinctAtomicAnswersArray[i].printMe();
		}

		/*
		 * The matrix of labelling decisions (D_ij) is created as a
		 * two-dimensional array of boolean values. (distinct atomicAnswers) X
		 * (no. of workers) The array contains the information if a worker has
		 * provided a distinct atomicAnswer or not. Originally, it should be an
		 * array of all possible labels. For the sake of simplicity we only
		 * consider assigned labels at this point.
		 * 
		 * i = number of labels (we will only use assigned labels for
		 * simplification at this point) j = number of participating workers
		 * 
		 * For each AtomicAnswer we check all the workers if they have submitted
		 * it. If yes, the entry is true (and we also count the total number of
		 * assigned labels)
		 */
		int[][] labellingDecisions = new int[allDistinctAtomicAnswersArray.length][workers.length];
		int totalNumberOfAssignedLabels = 0;
		int[] numberOfAssignmentsPerLabel = new int[allDistinctAtomicAnswersArray.length]; // keeping
																							// track
																							// of
																							// the
																							// total
																							// number
																							// of
																							// assigned
																							// labels
		for (int i = 0; i < allDistinctAtomicAnswersArray.length; i++) {
			numberOfAssignmentsPerLabel[i] = 0;
			for (int j = 0; j < workers.length; j++) {
				if (answers[j].getSetOfAtomicAnswers().contains(
						allDistinctAtomicAnswersArray[i])) {
					labellingDecisions[i][j] = 1;
					numberOfAssignmentsPerLabel[i] = numberOfAssignmentsPerLabel[i] + 1;
					totalNumberOfAssignedLabels++;
				} else {
					labellingDecisions[i][j] = 0;
				}
			}
			basiclog.info("[testing] Label " + (i + 1) + " was assigned "
					+ numberOfAssignmentsPerLabel[i] + " time(s).");
		}
		basiclog.info("[testing] The total number of assigned labels is "
				+ totalNumberOfAssignedLabels + ".");
		double averageNumberOfAssignedLabels = (double) totalNumberOfAssignedLabels
				/ workers.length; // can be used to estimate a prior; Attention:
									// there was an error: the ratio needs to be
									// casted to double!!
		basiclog.info("[testing] The average number of assigned labels assigned is "
				+ averageNumberOfAssignedLabels + ".");

		int number = allDistinctAtomicAnswersArray.length;
		int numberOfSubsets = (int) Math.pow(2, number);

		basiclog.info("[testing] There are 2 to the power of " + number
				+ ", i.e. " + numberOfSubsets
				+ " possible subsets of correct labels.");

		String[] binaryStringsArray = new String[numberOfSubsets];

		for (int i = 0; i < binaryStringsArray.length; i++) {
			String stringy = Integer.toBinaryString(i);
			for (int n = stringy.length(); n < number; n++) {
				stringy = "0" + stringy;
			}
			binaryStringsArray[i] = stringy;
		}

		int[][] binaryPermutations = new int[number][numberOfSubsets];

		for (int s = 0; s < numberOfSubsets; s++) {
			for (int i = 0; i < number; i++) {

				binaryPermutations[i][s] = Integer.parseInt(Character
						.toString(binaryStringsArray[s].charAt(i)));

			}

		}

		double[] probabilityOfSubsetsArray = new double[numberOfSubsets];
		double epsilon = 0;

		for (int t = 0; t < numberOfSubsets; t++) {

			double productIfLabelIsInThisSubset = 1; // because 1 is neutral
														// element of
														// multiplication
			double productIfLabelIsNotInThisSubset = 1;

			for (int i = 0; i < number; i++) {
				if (binaryPermutations[i][t] == 1) {
					for (int j = 0; j < workers.length; j++) {
						if (labellingDecisions[i][j] == binaryPermutations[i][t]) {
							productIfLabelIsInThisSubset = productIfLabelIsInThisSubset
									* sensitivity[j];
						} else {
							productIfLabelIsNotInThisSubset = productIfLabelIsNotInThisSubset
									* (1 - sensitivity[j]);
						}
					}

				} else {
					for (int j = 0; j < workers.length; j++) {
						if (labellingDecisions[i][j] == binaryPermutations[i][t]) {
							productIfLabelIsInThisSubset = productIfLabelIsInThisSubset
									* specificity[j];
						} else {
							productIfLabelIsNotInThisSubset = productIfLabelIsNotInThisSubset
									* (1 - specificity[j]);
						}
					}

				}
			}

			probabilityOfSubsetsArray[t] = productIfLabelIsInThisSubset
					* productIfLabelIsNotInThisSubset;
			epsilon = epsilon + probabilityOfSubsetsArray[t];

		}

		/*******************************************************************************
		 * Determining the subset with max probability and its index in the
		 * array
		 *******************************************************************************/

		double maxProbabilityOfAnySubset = probabilityOfSubsetsArray[0]
				/ epsilon;
		int indexOfSubsetWithMaxProbability = 0;

		for (int i = 0; i < probabilityOfSubsetsArray.length; i++) {
			probabilityOfSubsetsArray[i] = probabilityOfSubsetsArray[i]
					/ epsilon;
			basiclog.info("Subset " + (i + 1) + " (" + binaryStringsArray[i]
					+ ") has a probability of " + probabilityOfSubsetsArray[i]
					+ ".");
			if (probabilityOfSubsetsArray[i] >= maxProbabilityOfAnySubset) {
				maxProbabilityOfAnySubset = probabilityOfSubsetsArray[i];
				indexOfSubsetWithMaxProbability = i;
			}
		}

		basiclog.info("Epsilon is equal to " + epsilon + ".");
		basiclog.info("The most probable subset is subset "
				+ (indexOfSubsetWithMaxProbability + 1) + ", i.e. ("
				+ binaryStringsArray[indexOfSubsetWithMaxProbability]
				+ "), with probability " + maxProbabilityOfAnySubset + ".");

		// Computing the labelling performance in the case the subset mit max
		// probability gets accepted later
		// the subset is given by index

		for (int j = 0; j < workers.length; j++) {

			int tp = 0;
			int tn = 0;
			int fp = 0;
			int fn = 0;

			for (int i = 0; i < number; i++) {

				if (labellingDecisions[i][j] == 0
						&& binaryPermutations[i][indexOfSubsetWithMaxProbability] == 0) {
					tn++;
				} else if (labellingDecisions[i][j] == 0
						&& binaryPermutations[i][indexOfSubsetWithMaxProbability] == 1) {
					fn++;
				} else if (labellingDecisions[i][j] == 1
						&& binaryPermutations[i][indexOfSubsetWithMaxProbability] == 1) {
					tp++;
				} else if (labellingDecisions[i][j] == 1
						&& binaryPermutations[i][indexOfSubsetWithMaxProbability] == 0) {
					fp++;
				}
			}

			sensitivity[j] = (double) tp / (tp + fn);

			if (fp + tn == 0) {
				specificity[j] = (double) 1;
			} else {
				specificity[j] = (double) tn / (fp + tn);
			}
		}

		// define a line separator to be able to break lines
		String newline = System.getProperty("line.separator");

		basiclog.info("The significance threshold is " + phiMin + ".");

		if (maxProbabilityOfAnySubset > phiMin) {

			String xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
					+ newline
					+ "<QuestionFormAnswers xmlns=\"http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd\">";
			int correctCount = 1;
			for (int i = 0; i < number; i++) {

				if (binaryPermutations[i][indexOfSubsetWithMaxProbability] == 1) {
					basiclog.info("The label " + (i + 1)
							+ " was accepted for the final set.");

					ICDAtomicAnswer correctAtomicAnswer = allDistinctAtomicAnswersArray[i];
					xml = xml + newline + "<Answer><QuestionIdentifier>code"
							+ correctCount + "</QuestionIdentifier><FreeText>"
							+ correctAtomicAnswer.icdCode
							+ "</FreeText></Answer>" + newline + "<Answer>"
							+ newline + "<QuestionIdentifier>code"
							+ correctCount
							+ "flag1</QuestionIdentifier><FreeText>"
							+ correctAtomicAnswer.flag1 + "</FreeText>"
							+ newline + "</Answer>" + newline + "<Answer>"
							+ newline + "<QuestionIdentifier>code"
							+ correctCount
							+ "flag2</QuestionIdentifier><FreeText>"
							+ correctAtomicAnswer.flag2 + "</FreeText>"
							+ newline + "</Answer>" + newline + "<Answer>"
							+ newline + "<QuestionIdentifier>code"
							+ correctCount
							+ "flag3</QuestionIdentifier><FreeText>"
							+ correctAtomicAnswer.flag3 + "</FreeText>"
							+ newline + "</Answer>" + newline + "<Answer>"
							+ newline + "<QuestionIdentifier>code"
							+ correctCount
							+ "flag4</QuestionIdentifier><FreeText>"
							+ correctAtomicAnswer.flag4 + "</FreeText>"
							+ newline + "</Answer>";

					correctCount++;
				} else {
					basiclog.info("The label " + (i + 1)
							+ " was rejected for the final set.");
				}

			}
			xml = xml + newline + "</QuestionFormAnswers>";

			// Create new assignment and add the labels to it
			Assignment correctAssignment = new Assignment();
			if (correctCount != 1) {
				correctAssignment.setAnswer(xml);
				correctAssignment.setWorkerId("ICDQuality");
				correctAssignment.setAssignmentId("ICDQualityAssignment"
						+ new Date());
				correctAssignment.setHITId(hitID);
			}

			// do we have a single worker? or do we have more than one worker?
			// single: then do not update his sensitivity and specificity; his
			// #assignments++
			// multiple: then update their sensitivity and specificity. their
			// #assignments++
			if (workers.length < 2) {
				workers[0]
						.setTotalAssignments(workers[0].getTotalAssignments() + 1);
				basiclog.info("As only one worker was asked his/her labelling performance is not updated.");
			} else {
				for (int j = 0; j < workers.length; j++) {
					workers[j].addSensitivityValue(sensitivity[j]);
					workers[j].addSpecificityValue(specificity[j]);
					workers[j].setTotalAssignments(workers[j]
							.getTotalAssignments() + 1);
					basiclog.info("Worker " + (j + 1)
							+ " has a new sensitivity of " + sensitivity[j]
							+ " and a new specificity of " + specificity[j]
							+ ".");
				}
				basiclog.info("The labelling performance of participating workers has been updated.");
			}

			// finalize result and update worker statistics
			basiclog.info("Further redundancy is not required.");
			if (correctCount != 1) {
				Result correctResult = new Result(hitID, correctAssignment);
				correctResult.setCorrectSolution(AssignmentAnswerFactory
						.createInstance(correctAssignment.getAnswer()));
				correctResult.setFinalResult(true);
				ResultOutputCollector.getInstance().put(hitID, correctResult);
			}

		} // end if we have a result better than phiMin
		else {
			// if we have no result above phi significance

			basiclog.info("The most likely subset of labels has an unsufficient significance.");
			basiclog.info("We would require further redundancy. I am checking if we can increase redundancy or need to escalate HIT...");
			basiclog.info("The escalation threshold is " + epsilonMax
					+ " hoch " + number + " = " + Math.pow(epsilonMax, number)
					+ ".");

			if (epsilon > epsilonMax) {
				// publish one more task, i.e. extend hit
				basiclog.info("We do not need to escalate yet. I extend HIT instead.");
				basiclog.info("Done with HIT " + hitID
						+ ", extending HIT. (Phi_y="
						+ maxProbabilityOfAnySubset + ", Epsilon=" + epsilon
						+ ")\n");

				boolean success = HITManager.getInstance().extendHit(hitID);
				ResultOutputCollector.getInstance().put(hitID, r);
				int retryIn = 0;
				while (!success) {
					retryIn++;
					// retry
					success = HITManager.getInstance().extendHit(hitID);
					if (success) {
						basiclog.info("HIT " + hitID + " extended.");
						break;
					}
					basiclog.warn("Error while extending Hit " + hitID
							+ ". Retrying in " + retryIn + " seconds...");
					try {
						Thread.sleep(1000 * retryIn);
					} catch (InterruptedException e) {
						e.printStackTrace();
					}
				}
				ResultOutputCollector.getInstance().addRequestedAssignments(1);
			} else {
				// we need to escalate the hit
				basiclog.info("Unfortunately, we need to escalate this HIT.");
				basiclog.info("Done with HIT " + hitID
						+ ", escalating HIT. (Phi_y="
						+ maxProbabilityOfAnySubset + ", Epsilon=" + epsilon
						+ ")\n");

				ResultOutputCollector.getInstance().put(hitID, r);
				r.setCorrectSolution(null);
				r.setPhiCorrect(-2);
				ResultOutputCollector.getInstance().escalateHit(hitID);
			}

		}

	}
}
